{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<head>python爬虫基础3_xpath语法与lxml库的用法</head>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = '''\n",
    "<div>\n",
    "    <ul>\n",
    "         <li class=\"item-0\"><a href=\"link1.html\">第一段落</a></li>\n",
    "         <li class=\"item-1\"><a href=\"link2.html\">第二段落</a></li>\n",
    "         <li class=\"item-inactive\"><a href=\"link3.html\">第三段落</a></li>\n",
    "         <li class=\"item-1\"><a href=\"link4.html\">第四段落</a></li>\n",
    "         <li class=\"item-0\"><a href=\"link5.html\">第五段落</a>\n",
    "     </ul>\n",
    "</div>\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_text(text):\n",
    "    html = etree.HTML(text)\n",
    "    result = etree.tostring(html, encoding='utf-8').decode('utf-8')\n",
    "    return result\n",
    "\n",
    "# 解析html文件\n",
    "def parse_html(file_name):\n",
    "    parser = etree.HTMLParser(encoding='utf-8')\n",
    "    html_content = etree.parse(file_name, parser=parser)\n",
    "    return html_content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取class =“item-0”的数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### etree.HTML 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "首先我们使用 lxml 的 etree 库，然后利用 etree.HTML 初始化，然后我们将其打印出来。\n",
    "其中，这里体现了 lxml 的一个非常实用的功能就是自动修正 html 代码，\n",
    "大家应该注意到了，最后一个 li 标签，其实我把尾标签删掉了，是不闭合的。\n",
    "不过，lxml 因为继承了 libxml2 的特性，具有自动修正 HTML 代码的功能。\n",
    "'''\n",
    "html = etree.HTML(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据类型\n",
    "* 注只有lxml.etree._Element 类型时，才能使用xpath,str类型不能直接使用xpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lxml.etree._Element"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 显示内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element html at 0x2265037f808>\n"
     ]
    }
   ],
   "source": [
    "print(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html_text = etree.tostring(html, encoding='utf-8').decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<html><body><div>\\n    <ul>\\n         <li class=\"item-0\"><a href=\"link1.html\">第一段落</a></li>\\n         <li class=\"item-1\"><a href=\"link2.html\">第二段落</a></li>\\n         <li class=\"item-inactive\"><a href=\"link3.html\">第三段落</a></li>\\n         <li class=\"item-1\"><a href=\"link4.html\">第四段落</a></li>\\n         <li class=\"item-0\"><a href=\"link5.html\">第五段落</a>\\n     </li></ul>\\n</div>\\n</body></html>'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查找class为“item-0”的元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lis = html.xpath(\"//li[@class='item-0']\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义存储变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_dict = dict()\n",
    "text_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'title': '第一段落'}, {'title': '第五段落'}]\n"
     ]
    }
   ],
   "source": [
    "for li in lis:\n",
    "    text_dict['title'] = li.xpath(\"./a/text()\")[0]\n",
    "    text_list.append(text_dict)\n",
    "    text_dict = {}\n",
    "    \n",
    "print(text_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 只获取第一段落"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "html = etree.HTML(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lxml.etree._Element"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "li_text = html.xpath(\"//ul/li[1]/a/text()\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'第一段落'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "li_text "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 注xpath的html标识索引从1开始，而不是从0开始算"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取tr中class等于even的标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = parse_html(\"tencent.html\")\n",
    "trs = html.xpath(\"//tr[@class='even']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(len(trs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47428&amp;keywords=python&amp;tid=0&amp;lid=0\">30628-腾讯广告算法高级工程师（研发中心-深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>1</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47411&amp;keywords=python&amp;tid=0&amp;lid=0\">22989-腾讯云资深运营开发工程师（深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>2</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47379&amp;keywords=python&amp;tid=0&amp;lid=0\">22989-腾讯云serverless运营开发工程师（深圳总部）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>2</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47374&amp;keywords=python&amp;tid=0&amp;lid=0\">18436-NLP算法研究员（深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>1</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n"
     ]
    }
   ],
   "source": [
    "for tr in trs:\n",
    "    tr_content = etree.tostring(tr, encoding='utf-8').decode('utf-8')\n",
    "    print(tr_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取tr中class 包含even的标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = parse_html(\"tencent.html\")\n",
    "trs = html.xpath(\"//tr[contains(@class,'even')]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    }
   ],
   "source": [
    "print(len(trs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tr class=\"even active\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47441&amp;keywords=python&amp;tid=0&amp;lid=0\">TME-全民K歌数据产品经理</a></td>&#13;\n",
      "            <td>产品/项目类</td>&#13;\n",
      "            <td>2</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47428&amp;keywords=python&amp;tid=0&amp;lid=0\">30628-腾讯广告算法高级工程师（研发中心-深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>1</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47411&amp;keywords=python&amp;tid=0&amp;lid=0\">22989-腾讯云资深运营开发工程师（深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>2</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47379&amp;keywords=python&amp;tid=0&amp;lid=0\">22989-腾讯云serverless运营开发工程师（深圳总部）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>2</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n",
      "<tr class=\"even\">&#13;\n",
      "            <td class=\"l square\"><a target=\"_blank\" href=\"position_detail.php?id=47374&amp;keywords=python&amp;tid=0&amp;lid=0\">18436-NLP算法研究员（深圳）</a></td>&#13;\n",
      "            <td>技术类</td>&#13;\n",
      "            <td>1</td>&#13;\n",
      "            <td>深圳</td>&#13;\n",
      "            <td>2019-02-03</td>&#13;\n",
      "        </tr>&#13;\n",
      "                        \n"
     ]
    }
   ],
   "source": [
    "for tr in trs:\n",
    "    tr_content = etree.tostring(tr, encoding='utf-8').decode('utf-8')\n",
    "    print(tr_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取所有a标签下面的href属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "position_detail.php?id=47441&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47442&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47428&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47423&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47411&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47396&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47379&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47380&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47374&keywords=python&tid=0&lid=0\n",
      "position_detail.php?id=47359&keywords=python&tid=0&lid=0\n"
     ]
    }
   ],
   "source": [
    "html = parse_html(\"tencent.html\")\n",
    "a_list = html.xpath(\"//a/@href\")\n",
    "for a in a_list:\n",
    "    print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取所有职位信息(纯文本)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取所有的tr标签（不包含第一行）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = parse_html(\"tencent.html\")\n",
    "trs = html.xpath(\"//tr\")[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "print(len(trs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "html = parse_html(\"tencent.html\")\n",
    "trs = html.xpath(\"//tr[position()>1]\")\n",
    "# trs = html.xpath(\"//tr\")\n",
    "# print(len(trs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for tr in trs:\n",
    "#     tr_content = etree.tostring(tr, encoding='utf-8').decode('utf-8')\n",
    "#     print(tr_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TME-全民K歌数据产品经理\n",
      "TME-全民K歌高级数据产品经理\n",
      "30628-腾讯广告算法高级工程师（研发中心-深圳）\n",
      "TEG02-网络运维工程师\n",
      "22989-腾讯云资深运营开发工程师（深圳）\n",
      "PCG11-后台开发工程师（北京）\n",
      "22989-腾讯云serverless运营开发工程师（深圳总部）\n",
      "22989-腾讯云serverless运营开发工程师（成都）\n",
      "18436-NLP算法研究员（深圳）\n",
      "PCG17-QQ钱包后台开发工程师（深圳）\n"
     ]
    }
   ],
   "source": [
    "for tr in trs:\n",
    "    print(tr.xpath(\"./td/a/text()\")[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取所有td下面的（纯文本）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'职位名称': 'TME-全民K歌数据产品经理', '地点': '深圳', '职位类别': '产品/项目类', '人数': '2', '发布时间': '2019-02-03'}\n",
      "{'职位名称': 'TME-全民K歌高级数据产品经理', '地点': '深圳', '职位类别': '产品/项目类', '人数': '1', '发布时间': '2019-02-03'}\n",
      "{'职位名称': '30628-腾讯广告算法高级工程师（研发中心-深圳）', '地点': '深圳', '职位类别': '技术类', '人数': '1', '发布时间': '2019-02-03'}\n",
      "{'职位名称': 'TEG02-网络运维工程师', '地点': '深圳', '职位类别': '技术类', '人数': '1', '发布时间': '2019-02-03'}\n",
      "{'职位名称': '22989-腾讯云资深运营开发工程师（深圳）', '地点': '深圳', '职位类别': '技术类', '人数': '2', '发布时间': '2019-02-03'}\n",
      "{'职位名称': 'PCG11-后台开发工程师（北京）', '地点': '北京', '职位类别': '技术类', '人数': '1', '发布时间': '2019-02-03'}\n",
      "{'职位名称': '22989-腾讯云serverless运营开发工程师（深圳总部）', '地点': '深圳', '职位类别': '技术类', '人数': '2', '发布时间': '2019-02-03'}\n",
      "{'职位名称': '22989-腾讯云serverless运营开发工程师（成都）', '地点': '成都', '职位类别': '技术类', '人数': '2', '发布时间': '2019-02-03'}\n",
      "{'职位名称': '18436-NLP算法研究员（深圳）', '地点': '深圳', '职位类别': '技术类', '人数': '1', '发布时间': '2019-02-03'}\n",
      "{'职位名称': 'PCG17-QQ钱包后台开发工程师（深圳）', '地点': '深圳', '职位类别': '技术类', '人数': '1', '发布时间': '2019-02-03'}\n"
     ]
    }
   ],
   "source": [
    "position_dict = dict()\n",
    "position_list = []\n",
    "\n",
    "for tr in trs:\n",
    "    tds = tr.xpath(\"./td\")\n",
    "    position_dict[\"职位名称\"] = tds[0].xpath(\"./a/text()\")[0]\n",
    "    position_dict[\"职位类别\"] = tds[1].xpath(\"./text()\")[0]\n",
    "    position_dict[\"人数\"] = tds[2].xpath(\"./text()\")[0]\n",
    "    position_dict[\"地点\"] = tds[3].xpath(\"./text()\")[0]\n",
    "    position_dict[\"发布时间\"] = tds[4].xpath(\"./text()\")[0]\n",
    "    position_list.append(position_dict)\n",
    "    position_dict = {}\n",
    "# print(position_list)\n",
    "\n",
    "for pos in position_list:\n",
    "    print(pos)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存成csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "def write_csv_opt():\n",
    "    headers = [\"职位名称\", \"职位类别\", \"地点\",\"人数\",\"发布时间\"]\n",
    "    values = position_list\n",
    "    with open('position.csv', 'w', encoding='utf_8', newline='') as fp:\n",
    "        writer = csv.DictWriter(fp, headers)\n",
    "        # 写入表头数据时，需要调用writeheader方法\n",
    "        writer.writeheader()\n",
    "        writer.writerows(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## 此方法保存的csv直接用excel打开，中文乱码\n",
    "write_csv_opt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('position.csv', encoding='utf-8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>职位名称</th>\n",
       "      <th>职位类别</th>\n",
       "      <th>地点</th>\n",
       "      <th>人数</th>\n",
       "      <th>发布时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TME-全民K歌数据产品经理</td>\n",
       "      <td>产品/项目类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TME-全民K歌高级数据产品经理</td>\n",
       "      <td>产品/项目类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30628-腾讯广告算法高级工程师（研发中心-深圳）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEG02-网络运维工程师</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22989-腾讯云资深运营开发工程师（深圳）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PCG11-后台开发工程师（北京）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>北京</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>22989-腾讯云serverless运营开发工程师（深圳总部）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>22989-腾讯云serverless运营开发工程师（成都）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>成都</td>\n",
       "      <td>2</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>18436-NLP算法研究员（深圳）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PCG17-QQ钱包后台开发工程师（深圳）</td>\n",
       "      <td>技术类</td>\n",
       "      <td>深圳</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-02-03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               职位名称    职位类别  地点  人数        发布时间\n",
       "0                    TME-全民K歌数据产品经理  产品/项目类  深圳   2  2019-02-03\n",
       "1                  TME-全民K歌高级数据产品经理  产品/项目类  深圳   1  2019-02-03\n",
       "2        30628-腾讯广告算法高级工程师（研发中心-深圳）     技术类  深圳   1  2019-02-03\n",
       "3                     TEG02-网络运维工程师     技术类  深圳   1  2019-02-03\n",
       "4            22989-腾讯云资深运营开发工程师（深圳）     技术类  深圳   2  2019-02-03\n",
       "5                 PCG11-后台开发工程师（北京）     技术类  北京   1  2019-02-03\n",
       "6  22989-腾讯云serverless运营开发工程师（深圳总部）     技术类  深圳   2  2019-02-03\n",
       "7    22989-腾讯云serverless运营开发工程师（成都）     技术类  成都   2  2019-02-03\n",
       "8                18436-NLP算法研究员（深圳）     技术类  深圳   1  2019-02-03\n",
       "9             PCG17-QQ钱包后台开发工程师（深圳）     技术类  深圳   1  2019-02-03"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 中文乱码问题解决了\n",
    "df.to_csv(\"position2.csv\", encoding='utf_8_sig') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 似乎这样也是可以的\n",
    "def write_csv_opt_chs(file_name,headers,values):\n",
    "    with open(file_name, 'w', encoding='utf_8_sig', newline='') as fp:\n",
    "        writer = csv.DictWriter(fp, headers)\n",
    "        writer.writeheader()\n",
    "        writer.writerows(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "headers = [\"职位名称\", \"职位类别\", \"地点\",\"人数\",\"发布时间\"]\n",
    "write_csv_opt_chs(\"are_you_ok.csv\",headers,position_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 相邻元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html_tmp = \"\"\"\n",
    "<div>\n",
    "    <a id=\"1\" href=\"www.baidu.com\">我是第1个a标签</a>\n",
    "    <p>我是第一个p标签</p>\n",
    "    <a id=\"2\" href=\"www.baidu.com\">我是第2个a标签</a>\n",
    "    <a id=\"3\" href=\"www.baidu.com\">我是第3个a标签</a>\n",
    "    <a id=\"4\" href=\"www.baidu.com\">我是第4个a标签</a>\n",
    "    <p>我是第二个p标签</p>\n",
    "    <a id=\"5\" href=\"www.baidu.com\">我是第5个a标签</a>\n",
    "</div>\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查找第一个a标签下一个相邻（兄弟）的p标签\n",
    "\n",
    "* 获取第三个a标签后面的第N个标签：\"//a[@id='3']/following-sibling::*[N]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = etree.HTML(html_tmp)\n",
    "p_tag = html.xpath(\"//a[@id='1']/following-sibling::p[1]/text()\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['我是第一个p标签']"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_tag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查找第一个a标签相下一个邻的所有p标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我是第一个p标签']\n",
      "['我是第二个p标签']\n"
     ]
    }
   ],
   "source": [
    "html = etree.HTML(html_tmp)\n",
    "p_tags = html.xpath(\"//a[@id='1']/following-sibling::p\")\n",
    "for p in p_tags:\n",
    "    p_text = p.xpath(\"./text()\")\n",
    "    print(p_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取第三个a标签的上一个a标签：\n",
    "* 获取第三个a标签的前面的第N个标签：\"//a[@id='3']/preceding-sibling::*[N]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['我是第2个a标签']"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a2 = html.xpath(\"//a[@id='3']/preceding-sibling::a[1]/text()\")\n",
    "a2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "273px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
